import requests
import json
import re
from requests.auth import HTTPBasicAuth
import json
import time
import sys,os

from elasticsearch import Elasticsearch
from elasticsearch import helpers

from loguru import logger
from datetime import datetime,timedelta

import sys,os
from multiprocessing import Pool
from multiprocessing import Manager
import multiprocessing
from loguru import logger
from Neo4jServices import *
ES_INDEX_NAME = {
    "专利-180":"patent-180",
    "论文-180":"journal-180",
    "政策-180":"policies-180",
    "资讯-180":"news-180",
}

#名称字段
NAME_FIELD= {
    '专利-180':['patent_name','title'],
    "论文-180":["title","篇名"],
    "政策-180":["标题",'title'],
    "资讯-180":["标题","title"],
}

#ID字段
ID_FIELD = {
    "专利-180":["internal_id_elasticsearch","application_number","filename"],
    "论文-180":["internal_id_elasticsearch","resource_id"],
    "政策-180":['internal_id_elasticsearch',"自增id"],
    "资讯-180":['internal_id_elasticsearch',"自增id"]
}

AUTHOR_FIELD = {
    "专利-180":['author','inventor',"作者"],#是个列表
    "论文-180":["作者","author",'publisher'],
    "政策-180":[],
    "资讯-180":[],
}
ORGANIZATION_FIELD = {
    '专利-180':['organization','applicant_institution'],
    "论文-180":["机构","organization","institution"],
    "政策-180":["发布机构"],
    "资讯-180":[],
}
KEYWORDS_FIELD = {
    "专利-180":['keywords','keyword'],
    "论文-180":['keyword','keywords','中文关键词','复合关键词'],
    "政策-180":[],
    "资讯-180":[],
}
TIME_FIELD = {
    "专利-180":['application_date'],
    "论文-180":['publish_date','出版日期'],
    "政策-180":['时间','发布时间','发布日期'],
    "资讯-180":['时间'],
}
YEAR_FIELD = {
    "专利-180":['year'],
    "论文-180":['year',"年"],
    "资讯-180":[],
    "政策-180":[]
}
ABSTRACT_FIELD = {
    "专利-180":['abstract'],
    "论文-180":['abstract','中文摘要','中英文摘要'],
    "政策-180":['标题'],
    "资讯-180":['标题'],
}
FULLTEXT_FIELD = {
    "专利-180":[],
    "论文-180":[],
    "政策-180":['正文'],
    "资讯-180":['正文'],
}

def update_docs(es_instance,index="test",docs=[],id_to_unique='_id'):
    try:
        indexs_to_refresh = []
        actions = []
        for doc in docs:
            id_to_use = doc[id_to_unique]
            if id_to_unique =='_id':
                doc.pop('_id')
            doc_index = doc.pop('_index',None)
            if doc_index is None:
                doc_index = index
            indexs_to_refresh.append(doc_index)
            actions.append({"_index":doc_index,"doc":doc,"_id":id_to_use,"_op_type":"update"})
        successful, failed  = helpers.bulk(es_instance, actions)
        for in_ in list(set(indexs_to_refresh)):
            es_instance.indices.refresh(index=in_)
    except Exception as e:
        logger.error(f"Error update docs to index in Elasticsearch: {str(e)}.")
        raise e
    return successful, failed

#实体：专利（application_number）、标题（patent_name）、申请号（application_number）、公开日(publication_date）、申请日(application_date)、专利类型(patent_category)、公开国别(cp_name?)、权力要求数量（sovereignty_item?）、IPC（subject_code?）、国民经济类别(industry?）、摘要（abstract）、关键词（keyword）等12类。
#属性：公开 （公告） 号(publication_date？)、申请人（applicant_institution？）。
#关系：专利-标题、专利-申请号、专利-公开日、专利-申请日、专利-专利类型、专利-公开国别、专利-权力要求数量、专利-IPC、专利-国民经济类别、专利-摘要、摘要-关键词、引证(?)、被引证(?)、同族(?)等14种语义关系。
#问答展示图谱：除上述外，抽取abstract等内容的实体和关系。

def extract_patent_entity_relationship_and_save_to_neo4j(catelogy, search_tag,doc_data,neo4j_config,use_field_direct=True):
    """
    转存至 neo4j
    """
    index_name = ES_INDEX_NAME[catelogy]
    result = {}
    time_start_ai= time.time()
    #doc_data.pop('internal_id_elasticsearch',None)
    doc_data.pop('highlight',None)
    get_ = None
    #专利,
    for key in ID_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    patent= str(get_)
    if patent is None:
        logger.error("get doc {} patent is {}".format(doc_data,patent))
        return False
    doc_data['name'] = patent
    #标题
    get_ = None
    for key in NAME_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    patent_name = get_
    #申请号
    application_number = doc_data.get('application_number',doc_data.get('filename',None))
    #公开号,基本没有字段
    publication_num = doc_data.get('publication_number',doc_data.get('filename',doc_data.get('application_number',None)))
    #公开日
    publication_date = doc_data.get('publication_date',None)
    #申请日
    application_date = doc_data.get('application_date',None)
    #专利类型
    patent_category = doc_data.get('patent_category',None)
    #公开国别
    cp_name = doc_data.get('cp_name',None)
    #权力要求
    sovereignty_item = doc_data.get('sovereignty_item',None)
    #IPC
    subject_code = doc_data.get('subject_code',None)
    #国民经济类别
    industry = doc_data.get('industry',None)
    #概要
    abstract = doc_data.get('abstract',None)
    #关键词
    key_tags_list = []
    for keyword_name in ['chinese_keyword','machine_mark_keyword','machine_mark_keywords','keywords','keyword']:
        keys = doc_data.get(keyword_name,None)
        if keys:
            print('keys is',keys)
            for key in keys:
                key_tags = re.split(r'[、。/()（）,，；;]',key)
                key_tags_list = key_tags_list + key_tags
            logger.info("get keyword {} result {}".format(keyword_name,key_tags_list))
        else:
            logger.info("get keyword {} result None".format(keyword_name))
    key_tags_list = list(set(key_tags_list))
    logger.info("get keywords {}".format(key_tags_list))
    #申请人,有可能是str，也可能是list
    applicant_institution = doc_data.get('applicant_institution',doc_data.get("organization",doc_data.get('source',None)))
    #作者
    authors= doc_data.get('author',doc_data.get("inventor",None))
    province= doc_data.get('province',None)
    year_ = doc_data.get('year',None)
    if year_:
        if type(year_) is list:
            year_ = year_[0]
    #注意如果 "Unable to retrieve routing information",可能是Neo4j配置有问题，无法访问
    app = Neo4jServices(**neo4j_config)
    try:
        app.create_node("专利",{'name':patent})
        app.set_node_props("专利",patent,doc_data)
        app.create_node("行业标签_Level001",{'name':search_tag})
        app.create_relationship_between_nodes(patent,search_tag,"专利","行业标签_Level001","专利_行业标签_Level001")

        if use_field_direct==True:
            if year_:
                app.create_node("年份标签",{'name':year_})
                app.create_relationship_between_nodes(patent,year_,"专利","年份标签","专利_年份标签")
            if patent_name:
                app.create_node("标题",{'name':patent_name})
                app.create_relationship_between_nodes(patent,patent_name,"专利","标题","专利_标题")
            if publication_num:
                app.create_node("公开号",{'name':publication_num})
                app.create_relationship_between_nodes(patent,publication_num,"专利","公开号","专利_公开号")
            if application_number:
                app.create_node("申请号",{'name':application_number})
                app.create_relationship_between_nodes(patent,application_number,"专利","申请号","专利_申请号")
            if publication_date:
                app.create_node("公开日",{'name':publication_date})
                app.create_relationship_between_nodes(patent,publication_date,"专利","公开日","专利_公开日")
            if application_date:
                app.create_node("申请日",{'name':application_date})
                app.create_relationship_between_nodes(patent,application_date,"专利","申请日","专利_申请日")
            if applicant_institution:
                app.create_node("申请人",{'name':applicant_institution})
                app.create_relationship_between_nodes(patent,applicant_institution,"专利","申请人","专利_申请人")
            if authors:
                if type(authors) is list:
                    for au in authors:
                        app.create_node("作者",{'name':au})
                        app.create_relationship_between_nodes(patent,au,"专利","作者","专利_作者")
                else:
                    app.create_node("作者",{'name':authors})
                    app.create_relationship_between_nodes(patent,authors,"专利","作者","专利_作者")
            if patent_category:
                app.create_node("专利类型",{'name':patent_category})
                app.create_relationship_between_nodes(patent,patent_category,"专利","专利类型","专利_专利类型")
            if cp_name:
                app.create_node("公开国别",{'name':cp_name})
                app.create_relationship_between_nodes(patent,cp_name,"专利","公开国别","专利_公开国别")
            if sovereignty_item:
                app.create_node("权力要求数量",{'name':sovereignty_item})
                app.create_relationship_between_nodes(patent,sovereignty_item,"专利","权力要求数量","专利_权力要求数量")
            if subject_code:
                app.create_node("IPC",{'name':subject_code})
                app.create_relationship_between_nodes(patent,subject_code,"专利","IPC","专利_IPC")
            if industry:
                app.create_node("国民经济类别",{'name':industry})
                app.create_relationship_between_nodes(patent,industry,"专利","国民经济类别","专利_国民经济类别")
            if abstract:
                app.create_node("摘要",{'name':abstract})
                app.create_relationship_between_nodes(patent,abstract,"专利","摘要","专利_摘要")
            for key_tag in key_tags_list:
                app.create_node("关键词",{'name':key_tag})
                app.create_relationship_between_nodes(patent,key_tag,"专利","关键词","专利_关键词")
                if publication_date:
                    app.create_relationship_between_nodes(key_tag,publication_date,"关键词","公开日","关键词_公开日")
                if application_date:
                    app.create_relationship_between_nodes(key_tag,application_date,"关键词","申请日","关键词_申请日")
                if year_:
                    app.create_node("年份标签",{'name':year_})
                    app.create_relationship_between_nodes(key_tag,year_,"关键词","年份标签","关键词_年份标签")
                app.create_relationship_between_nodes(key_tag,search_tag,"关键词","行业标签_Level001","关键词_行业标签_Level001")
            if province:
                app.create_node("省份",{'name':province})
                app.create_relationship_between_nodes(patent,province,"专利","省份","专利_省份")
        time_end_ai= time.time()
        logger.debug("all time cost {}ms.".format(1000*(time_end_ai-time_start_ai)))
        return True
    except Exception as e:
        logger.error("Exception {} when doc {}".format(e,doc_data['name']))
    finally:
        app.close()
    time_end_ai= time.time()
    logger.debug("三元组 for {}, result {}, time cost {}ms.".format(abstract,result,1000*(time_end_ai-time_start_ai)))
    return False

def extract_entity_relationship_and_save_to_neo4j(catelogy, search_tag,doc_data,neo4j_config,use_field_direct=True):
    """
    """
    index_name = ES_INDEX_NAME[catelogy]
    if catelogy.endswith('-180'):
        catelogy_new = catelogy[:-4]
    else:
        catelogy_new = "新"+catelogy
    result = {}
    time_start_ai= time.time()
    doc_data.pop('highlight',None)
    get_ = None
    for key in ID_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    doc_id = get_
    if doc_id is None:
        logger.error("get doc {} id is {}".format(doc_data,doc_id))
        return False
    doc_id = str(doc_id)
    #以文档ID作为名字
    doc_data['name'] = doc_id
    #标题
    get_ = None
    for key in NAME_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    doc_title = get_
    #作者
    get_ = None
    for key in AUTHOR_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    doc_authors = get_
    #机构
    get_ = None
    for key in ORGANIZATION_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    doc_organizations= get_
    #关键词
    get_ = None
    for key in KEYWORDS_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    doc_keywords = get_
    #时间
    get_ = None
    for key in TIME_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    doc_time = get_
    #年份
    get_ = None
    for key in YEAR_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    doc_year = get_
    if doc_year:
        if type(doc_year) is list:
            doc_year= doc_year[0]

    #摘要
    get_ = None
    for key in ABSTRACT_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    doc_abstract = get_

    #全文
    get_ = None
    for key in FULLTEXT_FIELD[catelogy]:
        get_ = doc_data.get(key,None)
        if get_:
            break
    doc_fulltext = get_

    #关键词
    key_tags_list = []
    if doc_keywords:
        print('keys is',doc_keywords)
        if type(doc_keywords) is list:
            for key in doc_keywords:
                key_tags = re.split(r'[、。/()（）,，；;]',key)
                if '' in key_tags:
                   key_tags.remove('')
                key_tags_list = key_tags_list + key_tags
        else:
            key_tags = re.split(r'[、。/()（）,，；;]',doc_keywords)
            if '' in key_tags:
               key_tags.remove('')
            key_tags_list = key_tags_list + key_tags
        logger.info("get keyword {} result {}".format(doc_keywords,key_tags_list))
    else:
        logger.info("get keyword {} result None".format(doc_keywords))
    key_tags_list = list(set(key_tags_list))
    logger.info("get keywords {}".format(key_tags_list))
    #注意如果 "Unable to retrieve routing information",可能是Neo4j配置有问题，无法访问
    app = Neo4jServices(**neo4j_config)
    try:
        app.create_node(catelogy_new,{'name':doc_id})
        app.set_node_props(catelogy_new,doc_id,doc_data)
        app.create_node("行业标签_Level001",{'name':search_tag})
        app.create_relationship_between_nodes(doc_id,search_tag,catelogy_new,"行业标签_Level001",catelogy_new+"_行业标签_Level001")

        if use_field_direct==True:
            if doc_year:
                app.create_node("年份标签",{'name':doc_year})
                app.create_relationship_between_nodes(doc_id,doc_year,catelogy_new,"年份标签",catelogy_new+"_年份标签")
            if doc_title:
                app.create_node("标题",{'name':doc_title})
                app.create_relationship_between_nodes(doc_id,doc_title,catelogy_new,"标题",catelogy_new+"_标题")
            if doc_time:
                app.create_node("时间",{'name':doc_time})
                app.create_relationship_between_nodes(doc_id,doc_time,catelogy_new,"时间",catelogy_new+"_时间")
            if doc_abstract:
                app.create_node("摘要",{'name':doc_abstract})
                app.create_relationship_between_nodes(doc_id,doc_abstract,catelogy_new,"摘要",catelogy_new+"_摘要")
            if doc_fulltext:
                app.create_node("全文",{'name':doc_fulltext})
                app.create_relationship_between_nodes(doc_id,doc_fulltext,catelogy_new,"全文",catelogy_new+"_全文")
            for key_tag in key_tags_list:
                app.create_node("关键词",{'name':key_tag})
                app.create_relationship_between_nodes(doc_id,key_tag,catelogy_new,"关键词",catelogy_new+"_关键词")
                if doc_year:
                    app.create_node("年份标签",{'name':doc_year})
                    app.create_relationship_between_nodes(key_tag,doc_year,"关键词","年份标签","关键词_年份标签")
                app.create_relationship_between_nodes(key_tag,search_tag,"关键词","行业标签_Level001","关键词_行业标签_Level001")
            if doc_authors:
                if type(doc_authors) is list:
                    for author_ in  doc_authors:
                        app.create_node("作者",{'name':author_})
                        app.create_relationship_between_nodes(doc_id,author_,catelogy_new,"作者",catelogy_new+"_作者")
                else:
                    app.create_node("作者",{'name':doc_authors})
                    app.create_relationship_between_nodes(doc_id,doc_authors,catelogy_new,"作者",catelogy_new+"_作者")
            if doc_organizations:
                if type(doc_organizations) is list:
                    for doc_organizations_ in doc_organizations:
                        app.create_node("机构",{'name':doc_organizations_})
                        app.create_relationship_between_nodes(doc_id,doc_organizations_,catelogy_new,"机构",catelogy_new+"_机构")
                else:
                    app.create_node("机构",{'name':doc_organizations})
                    app.create_relationship_between_nodes(doc_id,doc_organizations,catelogy_new,"机构",catelogy_new+"_机构")
        time_end_ai= time.time()
        logger.debug("all time cost {}ms.".format(1000*(time_end_ai-time_start_ai)))
        return True
    except Exception as e:
        logger.error("Exception {} when doc {}".format(e,doc_data['name']))
    finally:
        app.close()
    time_end_ai= time.time()
    logger.debug("三元组 for {}, result {}, time cost {}ms.".format(doc_abstract,result,1000*(time_end_ai-time_start_ai)))
    return False

class multiProcessingHandler(multiprocessing.Process):
    def __init__(self,catelogy,search_tag,from_,step_size,c_number,doc_records_ids,doc_records_names,doc_failed_to_retry):
        multiprocessing.Process.__init__(self)
        self.catelogy = catelogy
        self.search_key = f'{search_tag}'+"_year_2014_to_2023"
        self.from_ = from_
        self.step_size = step_size
        self.c_number = c_number
        self.doc_records_ids=doc_records_ids
        self.doc_records_names = doc_records_names
        self.doc_failed_to_retry = doc_failed_to_retry

    def run(self):
        self.pid_record = os.getpid()
        catelogy = self.catelogy
        index_name=ES_INDEX_NAME[catelogy]
        search_key  = self.search_key
        from_ = self.from_
        step_size = self.step_size
        c_number = self.c_number
        es_instance = Elasticsearch(hosts=["https://119.254.155.105:19202"], request_timeout=3600, basic_auth=('elastic', 'EPPso10r8Ja'),verify_certs=False) #,ca_certs='/path/to/your/ca.pem')  # 如果有 CA 证书的路径，可以指定

        current_ids = []
        query = {}
        query_ = {
            "exists": {
                "field": f'{search_key}'
            },
        }
        query['sort'] = [
                {"internal_id_elasticsearch":{"order": "desc",'missing':"_last","unmapped_type":"long"}},
        ]
        query['query'] = query_
        query['track_total_hits']=True
        query['track_scores']=True
        query['size'] = step_size
        query['from'] = from_
        response = es_instance.search(index=index_name, body=query)

        max_size = response['hits']['total']['value']
        neo4j_config = {
                "uri":"neo4j://119.254.155.36:7687",
                "user":"neo4j",
                "password":"vPn**12@@56",
                "db_name":'neo4j'
            }
        response = es_instance.search(index=index_name, body=query)#,from_=i)
        total_hits = response['hits']['total']['value']
        docs = response['hits']['hits']
        docs_length = len(docs)
        logger.debug('PID {} 本次循环中返回的文档包括 {}个,c_number {}'.format(self.pid_record,docs_length,c_number))
        docs_to_add_fields = []
        count = 0
        ai_failure_count = 0
        ai_not_need_handle = 0
        for doc in docs:
            try:
                count = count+1
                time_start_for_one = time.time()
                logger.trace('--------->{}'.format(doc))
                doc_data =doc.pop('_source',None)
                get_ = None
                for key in ID_FIELD[catelogy]:
                    get_ = doc_data.get(key,None)
                    if get_:
                        break
                patent = str(get_)

                if patent is None:
                    self.doc_failed_to_retry.append(doc['_id'])
                    logger.error('PID {} 文档 {} 名称为{}.'.format(self.pid_record,doc['_id'],patent))
                    continue
                self.doc_records_names.append(patent)
                doc_data['_index_elasticsearch'] = doc['_index']
                doc_data['_id_elasticsearch'] = doc['_id']
                current_ids.append(doc['_id'])
                if doc_data.get('processed_by_ai_for_patent_neo4j_2025_5_n_n',None):
                    ai_not_need_handle = ai_not_need_handle+1
                    logger.info("PID {} doc {} has been processed, current progress is {}/{},ai_failure_count {},ai_not_need_handle {},max_size {},total_hits {},c_number {}.".format(self.pid_record,doc['_id'],count,docs_length,ai_failure_count,ai_not_need_handle,max_size,total_hits,c_number))
                    continue
                if catelogy == '专利-180':
                    flag_r = extract_patent_entity_relationship_and_save_to_neo4j(catelogy=catelogy,search_tag=search_tag,doc_data=doc_data,neo4j_config=neo4j_config,use_field_direct=True)
                else:
                    flag_r = extract_entity_relationship_and_save_to_neo4j(catelogy=catelogy, search_tag=search_tag,doc_data=doc_data,neo4j_config=neo4j_config,use_field_direct=True)
                logger.info('PID {} 抽取{} 结果为{}'.format(self.pid_record,doc['_id'],flag_r))
                update_doc = {}
                update_doc['_id'] = doc['_id']
                update_doc['_index'] = doc['_index']
                update_doc['processed_by_ai_for_patent_neo4j_2025_5_n_n'] =True
                if flag_r:
                    docs_to_add_fields.append(update_doc)
                else:
                    ai_failure_count = ai_failure_count+1
                    logger.info('PID {} 抽取{} 结果为{},不更新处理标记'.format(self.pid_record,doc['_id'],flag_r))
                time_end_for_one = time.time()
                time_cost_for_one = 1000*(time_end_for_one-time_start_for_one)
                logger.info("PID {} Add Flag Processed in API,current progress is {}/{},ai_failure_count {},ai_not_need_handle {},max_size {},total_hits {},c_number {},cost {}ms ".format(self.pid_record,count,docs_length,ai_failure_count,ai_not_need_handle,max_size,total_hits,c_number,time_cost_for_one))
            except Exception as e:
                self.doc_failed_to_retry.append(doc['_id'])
                logger.error('PID {} Exception {} For doc {}'.format(self.pid_record,e,doc['_id']))
                continue
        successful = 0
        failed = 0
        logger.debug("PID {} data is {}, search_tag {},catelogy {}".format(self.pid_record,current_ids,search_tag,catelogy))
        logger.info("PID {} add flag save_to_es index {} for data length {}/{},ai_failure_count {},ai_not_need_handle {},total_hits {},c_number {}".format(self.pid_record,index_name, len(docs_to_add_fields),max_size,ai_failure_count,ai_not_need_handle,total_hits,c_number))
        successful, failed  =  update_docs(es_instance,index_name,docs_to_add_fields)
        self.doc_records_ids= self.doc_records_ids+ current_ids
        logger.info("PID {} add flag save_to_es index {} for data length {} successed {},failed {},ai_failure_count {},ai_not_need_handle {},c_number {}".format(self.pid_record,index_name, len(docs_to_add_fields),successful,failed,ai_failure_count,ai_not_need_handle,c_number))

es_instance = Elasticsearch(hosts=["https://119.254.155.105:19202"], timeout=3600, http_auth=('elastic', 'EPPso10r8Ja'),verify_certs=False) #,ca_certs='/path/to/your/ca.pem')  # 如果有 CA 证书的路径，可以指定
doc_records_ids =  multiprocessing.Manager().list()
doc_records_names =  multiprocessing.Manager().list()
doc_failed_to_retry = multiprocessing.Manager().list()
for search_tag in ['人参','碳纤维']:
    for catelogy in ['专利-180','资讯-180','论文-180','政策-180']:
        index_name=ES_INDEX_NAME[catelogy]
        #清空 query
        processes = []
        search_key = f'{search_tag}'+"_year_2014_to_2023"
        query = {}
        query_ = {
            "exists": {
                "field": f'{search_key}'
            },
        }
        query['sort'] = [
                {"internal_id_elasticsearch":{"order": "desc",'missing':"_last","unmapped_type":"long"}},
        ]
        query['track_scores']=True
        query['track_total_hits']=True
        query['query'] = query_
        query['size'] = 0
        response = es_instance.search(index=index_name, body=query)
        max_size = response['hits']['total']['value']
        step_size = 5000
        c_number = 0
        for i in range(0, max_size, step_size):
            c_number = c_number+1
            from_ = i
            #开启子进程进行处理并等待结束
            m = multiProcessingHandler(catelogy,search_tag,from_,step_size,c_number,doc_records_ids,doc_records_names,doc_failed_to_retry)
            processes.append(m)
        tmp_count = 0
        for process in processes:
            tmp_count=tmp_count+1
            logger.debug("Process {} Started.".format(tmp_count))
            process.daemon = True
            process.start()
        for process in processes:
            process.join()
